package cn.edu.nju.ws.sview.crawler;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;

import org.apache.log4j.Logger;

import cn.edu.nju.ws.sview.cache.RelationCache;
import cn.edu.nju.ws.sview.cache.URIIDCache;
import cn.edu.nju.ws.sview.database.DBConnectionFactory;
import cn.edu.nju.ws.sview.rdf.ResourceFactory;
import cn.edu.nju.ws.sview.rdf.URIResource;
import cn.edu.nju.ws.sview.rdf.canonicalization.Canonicalizer;
import cn.edu.nju.ws.sview.rdf.canonicalization.comparator.LabeledResourceComparator;
import cn.edu.nju.ws.sview.rdf.canonicalization.comparator.LiteralComparator;
import cn.edu.nju.ws.sview.rdf.canonicalization.comparator.StringComparator;
import cn.edu.nju.ws.sview.rdf.canonicalization.labeled.LabeledRDFGraph;
import cn.edu.nju.ws.sview.rdf.sentence.RDFSentence;
import cn.edu.nju.ws.sview.rdf.sentence.RDFSentenceParser;
import cn.edu.nju.ws.sview.util.LogUtil;
import cn.edu.nju.ws.sview.util.MD5;

import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.vocabulary.OWL;

/**
 * 完成爬虫中RDF文档的存储和索引
 * @author Gong Saisai
 * 
 * 关联表格 rdf_doc, uri,quadruple
 * 
 */
public class Store {
	static Logger logger = LogUtil.getLogger(Store.class.getName()) ;
	/*store all the uri in the model*/
	private HashSet<String> uriSet;
	/*store all the blanknode in the model*/
	private HashMap<Resource,Integer> bnode2ID;
	private HashMap<String,Integer> uri2ID;
	
    public Store(){
    	this.uriSet = new HashSet<String>();
		this.bnode2ID = new HashMap<Resource,Integer>();
		this.uri2ID = new HashMap<String,Integer>();
    }
    
    public void reset(){
    	this.uriSet.clear();
    	this.bnode2ID.clear();
    	this.uri2ID.clear();
    }
    
    
    /**
     * 更新RDF文档，如果该RDF文档存在旧版本则先删除再写入新版本
	 *     否则直接查询新版本
     * @param uri 待更新的文档URI
     * @param ontModel 装载文档内容的Model
     * @param timestamp 文档下载的时间戳
     * @throws Exception 
     */
	public void updateRDFDocument(URIResource uri,OntModel ontModel, long timestamp) throws Exception {
		int id =this.existDoc(uri.getURI()) ;
		//!!!!!!!包含旧版本的文档，直接删除，而不是通过
		//文档内容hash是否相同来判断是否删除，默认URI表不删除内容
		if(id!=-1){
			//删除三元组表中的信息
			this.deleteQuadruple(uri.getURI()) ;
			//删除RDF文档timestamp信息
			this.deleteDocTimeStamp(uri.getURI());
			//删除URI的dereference信息
			this.deleteDereferenceInfo(uri.getURI()) ;
		}
		this.reset();
		//分配ID，包括文档的
		this.allocateURIID(ontModel,uri.getURI());
		//插入四元组
		this.addQuadruple(ontModel,uri.getURI());
		//加入RDF文档timestamp
		this.addDocTimeStamp(timestamp, uri.getURI());
		//扫描所有relation，将可用的记入rs_ds表
		RelationCache.getInstance().countAvailableRelations(uri.getID());
	}
    
	/**
	 * 如果存在RDF文档，则将它相关的内容删除
	 * @param uri 文档URI
	 * @throws SQLException 
	 */
	public void deleteRDFDocument(URIResource uri) throws SQLException{
		int id =this.existDoc(uri.getURI()) ;
		
		//!!!!!!!包含旧版本的文档，直接删除，而不是通过
		//文档内容hash是否相同来判断是否删除，默认URI表不删除内容
		if(id!=-1){
			//删除三元组表中的信息
			this.deleteQuadruple(uri.getURI()) ;
			//删除RDF文档timestamp信息
			this.deleteDocTimeStamp(uri.getURI());
			//删除URI的dereference信息
			this.deleteDereferenceInfo(uri.getURI()) ;
			//删除URI与relation的available关系
			RelationCache.getInstance().deleteAvailableRelations(uri.getID());
		}
	}

	/**
	 * 向URI表插入内容，結束時uri2ID中装满了URI到数据库ID的对应关系
	 * @param model 装载数据的模型
	 * @param con 数据库连接
	 * @param uri 文档URI
	 * @throws Exception 
	 */
	private void allocateURIID(OntModel model,String doc_uri) throws Exception{
		try {
			
			/*get all the uris and blanknodes*/
			//为文档URI分配ID
			this.uriSet.add(doc_uri);
			StmtIterator stmtIt = model.listStatements();
			int rowNum = 0;// 元组数目
			NextTriple: while (stmtIt.hasNext()) {
				rowNum++;
				Statement stmt = stmtIt.nextStatement();
				Resource subject = stmt.getSubject();
				if (subject.isURIResource()) {
					this.uriSet.add(subject.toString());
				} else if (subject.isAnon()) {
					int id = this.getBnodeID(subject);
					if (id == -1) {
						int length = bnode2ID.size();
						this.bnode2ID.put(subject, length + 1);
					}
				} else {
					continue NextTriple;
				}
				Resource predicate = stmt.getPredicate();
				if (predicate.isURIResource()) {
					this.uriSet.add(predicate.toString());
				} else {
					continue NextTriple;
				}
				RDFNode object = stmt.getObject();
				if (object.isURIResource()) {
					this.uriSet.add(object.toString());
				} else if (object.isAnon()) {
					int id = this.getBnodeID((Resource) object);
					if (id == -1) {
						int length = this.bnode2ID.size();
						this.bnode2ID.put((Resource) object, length + 1);
					}
				} else if (object.isLiteral()) {
					String datatypeURI = ((Literal) object).getDatatypeURI();
					if (datatypeURI != null) {
						this.uriSet.add(datatypeURI);
					}
				} else {
					continue NextTriple;
				}
			}/*end of get all uris and blanknodes*/
			
		
			Iterator<String> iterator = this.uriSet.iterator();
			while (iterator.hasNext()) {
				String uri = iterator.next();
				int uri_id = -1;
				try {
					uri_id = URIIDCache.getInstance().getURIID(uri, true);
				} catch (Throwable e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				/*uri_id=-1 意味着之后与它相关的三元组全部丢弃 
				 * uri_id=-1其实是因为mysql 5.0/5.1不能存储4个字节的utf-8
				 * */
				this.uri2ID.put(uri, new Integer(uri_id));
				/*cache the uri to id mapping
				 * to prevent remove by whirlycache management
				 * */
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	/**
	 * 插入四元组
	 * @param model 该文档数据模型
	 * @param con 数据库连接
	 * @param doc_uri 文档URI
	 * @throws Exception 
	 */
	public void addQuadruple(OntModel model, String doc_uri) throws Exception{
		Connection con = DBConnectionFactory.getConnection();
		int BatchSize = 500;
		int count = 0;
		PreparedStatement  ps_triple2 = null;
		try {
			con.setAutoCommit(false);
			ps_triple2 = con
					.prepareStatement("insert into quadruple(s,p,o,datatypeid,lang,doc_id,sentence_hash) values(?,?,?,?,?,?,?)");
			ArrayList<RDFSentence> sentences = (new RDFSentenceParser(model))
					.getSentences();
			for (RDFSentence sentence : sentences) {
				byte[] sentence_hash = this.computeSentencHash(sentence) ;
				ArrayList<Statement> stmt_array = sentence.getStatements();
				for (Statement stmt : stmt_array) {
					count++;
					String s = "";
					String p = "";
					String o = "";
					String lang = null;
					int datatypeId = 0;
					Resource subject = stmt.getSubject();
					if (subject.isURIResource()) {
						int uri_id = this.getURIId(subject.toString());
						//意味着mysql无法存储4位utf-8
						if(uri_id<=0){
							continue;
						}
						s = "u" + uri_id;
					} else if (subject.isAnon()) {
						s = "b" + this.getURIId(doc_uri) + ":"
								+ (this.bnode2ID.get(subject)).intValue();
					}
					Resource predicate = stmt.getPredicate();
					if (predicate.isURIResource()) {
						int uri_id = this.getURIId(predicate.toString());
						//意味着mysql无法存储4位utf-8
						if(uri_id<=0){
							continue;
						}
						p = "u" + uri_id;
					}
					RDFNode object = stmt.getObject();
					if (object.isURIResource()) {
						int uri_id = this.getURIId(((Resource) object)
								.toString());
						//意味着mysql无法存储4位utf-8
						if(uri_id<=0){
							continue;
						}
						o = "u" + uri_id;
					} else if (object.isAnon()) {
						o = "b"
								+ this.getURIId(doc_uri)
								+ ":"
								+ (this.bnode2ID.get((Resource) object))
										.intValue();
					} else if (object.isLiteral()) {
						Literal literal = (Literal) object;
						String lex = literal.getLexicalForm();
						o = "l" + lex;
						lang = literal.getLanguage();
						if (literal.getDatatypeURI() != null) {
							datatypeId = this.uri2ID.get(literal
									.getDatatypeURI());
						}
					}
					ps_triple2.setString(1, s);
					ps_triple2.setString(2, p);
					ps_triple2.setString(3, o);
					if (datatypeId > 0) {
						ps_triple2.setInt(4, datatypeId);
					} else
						ps_triple2.setObject(4, null);
					ps_triple2.setString(5, lang);
					ps_triple2.setInt(6, this.getURIId(doc_uri));
					ps_triple2.setBytes(7, sentence_hash);
					ps_triple2.addBatch();
					if (count >= BatchSize) {
						ps_triple2.executeBatch();
						con.commit();
						ps_triple2.clearBatch();
						count = 0;
					}
				}
			}
			if (count >0) {
				ps_triple2.executeBatch();
				con.commit();
				ps_triple2.clearBatch();
				count = 0;
			}
			
			/*get owl:imports and download*/
			con.setAutoCommit(true) ;
			PreparedStatement stmt = con.prepareStatement("select o from quadruple where p=? and doc_id=? and o like 'u%'");
			try {
				int pid = URIIDCache.getInstance().getURIID(OWL.imports.getURI(), true) ;
				String puri = "u"+pid;
				stmt.setString(1,puri) ;
				stmt.setInt(2, URIIDCache.getInstance().getURIID(doc_uri, false)) ;
				ResultSet rs = stmt.executeQuery();
				while(rs.next()){
					int oid  = Integer.parseInt(rs.getString("o").substring(1)) ;
					String ostr = URIIDCache.getInstance().getURI(oid) ;
					Crawler.getInstance().submitURI(ResourceFactory.getInstance().createURIResource(ostr)) ;
				}
				rs.close();
			} catch (Throwable e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
			stmt.close();
			ps_triple2.close();
			con.close();
		}catch (Exception e) {
			e.printStackTrace();
			// 下面的东西十分重要，必须写上
			if (con != null) {
				try {
					con.rollback();
				} catch (SQLException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
				con.close();
			}
			throw new Exception("addQuadruple error");
		}
	}
	
	/**
	 * 删除三元组表中的信息
	 * @param con
	 * @param doc_uri 文档URI
	 * @throws SQLException 
	 */
	public void deleteQuadruple(String doc_uri) throws SQLException{
		Connection con = DBConnectionFactory.getConnection();
		PreparedStatement ps_triple = con
				.prepareStatement("delete from quadruple where doc_id=?");
		int id = -1;
		if (this.uri2ID.containsKey(doc_uri))
			id = this.uri2ID.get(doc_uri);
		else
			id = this.getDocId(doc_uri);
		ps_triple.setInt(1, id);
		ps_triple.executeUpdate();
		ps_triple.close();
		con.close();
	}
	
	/**
	 * @param res
	 * @return -1 if Bnode does not exist
	 */
	private int getBnodeID(Resource res) {
		Integer id = this.bnode2ID.get(res);
		if (id == null) {
			return -1;
		}
		return id.intValue();
	}
	
	private int getURIId(String uri){
		Integer id = uri2ID.get(uri);
		if(id==null){
			try {
				id = new Integer(URIIDCache.getInstance().getURIID(uri, true)) ;
			} catch (Throwable e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		if(id!=null)
			return id.intValue();
		else 
			return -1;
	}
	
	/**
	 * 通过timestamp那张表来判断是否有旧版本的RDF文档存在，如果存在返回ID
	 * @param con
	 * @param uri 文档URI
	 * @return 返回-1表示不存在
	 */
	private int existDoc(String uri){
		int id = -1;
		try {
			Connection con = DBConnectionFactory.getConnection();
			id = this.getDocId(uri);
			PreparedStatement ps = con
					.prepareStatement("select doc_id from quadruple where doc_id=? limit 1");
			ps.setInt(1, id);
			ResultSet rs = ps.executeQuery();
			if (!rs.next())
				id = -1;
			con.close();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return id;
	}
	
	/**
	 * 记录文档下载时间
	 * @param con
	 * @param uri
	 * @throws SQLException 
	 */
	public void addDocTimeStamp(long timestamp,String uri) throws SQLException{
		Connection con = DBConnectionFactory.getConnection();
		PreparedStatement ps;

		ps = con.prepareStatement("insert  into rdf_doc(id,timestamp) values(?,?) on duplicate key update timestamp=values(timestamp)");
		int id = -1;
		if (this.uri2ID.containsKey(uri))
			id = this.uri2ID.get(uri);
		else
			id = this.getDocId(uri);
		ps.setInt(1, id);
		ps.setLong(2, timestamp);
		ps.executeUpdate();
		ps.close();
		con.close();
	}
	
	/**
	 * 删除文档下载时间表中的数据
	 * @param con
	 * @param uri
	 * @throws SQLException 
	 */
	private void deleteDocTimeStamp(String uri) throws SQLException {
		// TODO Auto-generated method stub
		Connection con = DBConnectionFactory.getConnection();
			PreparedStatement  ps_triple = con.prepareStatement("delete from rdf_doc where id=?");
			int id = -1;
			if(this.uri2ID.containsKey(uri))
				id = this.uri2ID.get(uri) ;
			else
				id =this.getDocId(uri) ;
			ps_triple.setInt(1,id) ;
			ps_triple.executeUpdate() ;
			ps_triple.close();
		con.close();

	}
	
	/**
	 * 在数据库中加入URI的dereference信息
	 * @param con 数据库连接
	 * @param uri URI
	 * @param docURI 权威文档URI
	 * @throws SQLException 
	 */
	public  void setDereferenceInfo(String uri, String docURI) throws SQLException{
		Connection con = DBConnectionFactory.getConnection();
		
		PreparedStatement ps;
		ps = con.prepareStatement("update uri set deref=? where id =?");
		int uri_id = -1;
		if (this.uri2ID.containsKey(uri))
			uri_id = this.uri2ID.get(uri);
		else
			uri_id = this.getDocId(uri);
		int doc_id = -1;
		if (this.uri2ID.containsKey(docURI))
			doc_id = this.uri2ID.get(docURI);
		else
			doc_id = this.getDocId(docURI);
		ps.setInt(1, doc_id);
		ps.setInt(2, uri_id);
		ps.executeUpdate();
		ps.close();
        
		con.close();
	}
	
	/**
	 * 在数据库中删除URI的dereference信息
	 * @param con
	 * @param docURI 权威文档URI
	 * @throws SQLException 
	 */
	public void deleteDereferenceInfo(String docURI) throws SQLException{
		Connection con = DBConnectionFactory.getConnection();
		PreparedStatement ps_triple = con
				.prepareStatement("update uri set deref=0 where id =?");
		int id = -1;
		if (this.uri2ID.containsKey(docURI))
			id = this.uri2ID.get(docURI);
		else
			id = this.getDocId(docURI);
		ps_triple.setInt(1, id);
		ps_triple.executeUpdate();
		ps_triple.close();
		con.close();
	}
	
	//得到文档ID
	public int getDocId(String uri){
		int id = -1;
	    id = this.getURIId(uri) ;
	    return id;
	}
	
	private byte[] computeSentencHash(RDFSentence sentence){
		StringComparator stringComparator = new StringComparator();
		LiteralComparator literalComparator = new LiteralComparator(
				stringComparator);
		LabeledResourceComparator LabeledResourceComparator = new LabeledResourceComparator(
				stringComparator, literalComparator);
		Canonicalizer canonicalizer = new Canonicalizer(
				LabeledResourceComparator);
		LabeledRDFGraph labeledSentence = new LabeledRDFGraph(sentence
				.getStatements());
		labeledSentence.construct();
		MD5 sentenceMD5 = new MD5();
		byte[] sentence_hash = canonicalizer.canonicalizeRDFGraph(labeledSentence,
				sentenceMD5);
		return sentence_hash;
	}

	/**
	 * set last ping time and expire time
	 * @param uri
	 * @param timestamp
	 */
	public static void setLastPingTime(String uri, long timestamp) {
		// TODO Auto-generated method stub
		try {
			Connection con = DBConnectionFactory.getConnection();
			PreparedStatement ps = con.prepareStatement("update uri set lastping=?, expire=? where uri=? and md5=?");
			ps.setLong(1, timestamp) ;
			long expireTime = timestamp + Crawler.RDF_DOC_EXPIRE_TIME;
			ps.setLong(2, expireTime);
			ps.setString(3, uri) ;
			ps.setBytes(4, (new MD5()).makeMD5(uri)) ;
			int res = ps.executeUpdate();
			ps.close();
			con.close();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}
